package com.co.webmagic.process;

import com.alibaba.fastjson.JSON;
import com.co.common.util.IdWorker;
import com.co.video.pojo.Detail;
import com.co.video.pojo.Video;
import com.co.video.pojo.VideoState;
import com.co.video.service.CategoryService;
import com.co.video.service.DetailService;
import com.co.video.service.VideoService;
import com.co.video.service.VideoStateService;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.Date;
import java.util.List;
import java.util.Random;

/**
 * @author ldl.plus
 * @date 2020年06月30日  23:08
 */
@Component
public class AcFunJobProcessor implements PageProcessor {

    @Autowired
    private DetailService detailService;

    @Autowired
    private VideoService videoService;

    @Autowired
    private VideoStateService videoStateService;

    @Autowired
    private CategoryService categoryService;

    @Autowired
    private IdWorker idWorker;
    private static final Logger log = LoggerFactory.getLogger(AcFunJobProcessor.class);

    /**
     * 解析页面
     *
     * @param page
     */
    @Override
    @Transactional
    public void process(Page page) {
        HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
        httpClientDownloader.setProxyProvider(SimpleProxyProvider.from(
                new Proxy("163.125.235.218", 8118),
                new Proxy("27.43.190.19", 9999),
                new Proxy("183.166.103.35", 9999),
                new Proxy("125.73.220.18", 49128)));

        List<Selectable> list = page.getHtml().css("div.card-list div.search-video-card").nodes();
        if (list.size() == 0) {
            // 详情页，保存相关信息
            this.saveJobInfo(page);
        } else {
            // 列表页，解析出详情页URL地址，放入URL队列中
            for (Selectable selectable : list) {
                String url = selectable.css("div.video__cover>a", "href").get();
                String image = selectable.css("div.video__cover>a>img", "src").get();
                String time = selectable.css("div.video__cover>a>span", "text").get();
                int times = Integer.parseInt(time.split(":")[0]) * 60 + Integer.parseInt(time.split(":")[1]);
                String title = selectable.css("div.video__main__title>a", "text").get();
                String username = selectable.css("span.user-name", "text").get();

                Detail detail = new Detail();
                detail.setAid(idWorker.nextId() + "");
                detail.setTitle(title);
                detail.setImage(image);
                detail.setUrl(urlPre + url);
                detail.setUploadTime(new Date());
                detail.setDurations(times);
                detail.setStatus(1);
                detail.setUsername(username);
                detail.setIsUpload(1);
                detail.setVideos(1);
                detail.setCreateTime(new Date());
                detail.setUpdateTime(new Date());

                detailService.add(detail);

                page.addTargetRequest(detail.getUrl());
            }

            // 获取下一页的选择器
            page.addTargetRequest(searchUrl + "&sortType=1&channelId=0" +
                    "&requestId=N18wXzE1OTM1OTg4NjU0NzZf5a2m5LmgXzI2MA&pCursor=" + count++);
            if (count == 7) {
                System.exit(0);
            }
        }


    }

    public static final String searchUrl = "https://www.acfun.cn/search?type=video&keyword=python";

    /**
     * 解析页面，保存相关数据
     *
     * @param page
     */
    private void saveJobInfo(Page page) {
        Html html = page.getHtml();
        String title = html.css("h1.title span", "text").get();
        Detail detail = detailService.findByTitle(title);
        if (detail == null) {
            // 前面添加过视频图片的
            return;
        }
        List<Video> videoList = videoService.findByAid(detail.getAid());
        if (videoList != null && videoList.size() > 0) {
            // 添加过相应视频的跳过，不重复添加
            return;
        }
        String comment = html.css("div.description-container", "text").get();
        if (StringUtils.isNotEmpty(comment)) {
            detail.setComment(comment);
            detailService.update(detail);
        }
        // 新的一条数据
        List<Selectable> list = html.css("ul.scroll-div.over-parts li.single-p").nodes();
        if (list.size() == 0) {
            // 只有一个视频
            Video video = new Video();
            video.setCid(idWorker.nextId() + "");
            video.setAid(detail.getAid());
            video.setPage(1);
            video.setFrom("crawler");
            video.setPart(title);
            // String duration = html.css("span[data-bind-key='duration']", "text").get();
            video.setDuration(detail.getDurations());
            video.setUrl(detail.getUrl());
            video.setCreateTime(new Date());
            video.setUpdateTime(new Date());

            videoService.add(video);
            log.info(JSON.toJSONString(video));
        } else {
            // 有多个视频
            for (int i = 1; i <= list.size(); i++) {
                Selectable selectable = list.get(i - 1);
                Video video = new Video();
                video.setCid(idWorker.nextId() + "");
                video.setAid(detail.getAid());
                video.setPage(i);
                video.setFrom("crawler");
                title = selectable.css("li[title]", "title").get();
                video.setPart(title);
                video.setDuration(detail.getDurations());
                String url = selectable.css("li[data-href]", "data-href").get();
                video.setUrl(urlPre + url);
                video.setCreateTime(new Date());
                video.setUpdateTime(new Date());
                videoService.add(video);
                log.info(JSON.toJSONString(video));
            }
        }

        VideoState videoState = new VideoState();
        Random random = new Random();
        videoState.setView(random.nextInt(10000));
        videoState.setReply(random.nextInt(10000));
        videoState.setDanmaku(random.nextInt(10000));
        videoState.setFavorite(random.nextInt(10000));
        videoState.setLike(random.nextInt(10000));
        videoState.setDislike(random.nextInt(10000));
        videoState.setNowRank(random.nextInt(10000));
        videoState.setHisRank(random.nextInt(10000));
        videoState.setShare(random.nextInt(10000));
        videoState.setId(idWorker.nextId() + "");
        videoState.setAid(detail.getAid());
        videoState.setCreateTime(new Date());
        videoState.setUpdateTime(new Date());

        videoStateService.add(videoState);

        log.info(JSON.toJSONString(detail));
        log.info(JSON.toJSONString(videoState));
    }

    /**
     * 重试次数和抓取间隔
     */
    private final Site site = Site.me()
            .setCharset("UTF8")
            .setTimeOut(3000)
            .setRetrySleepTime(1000)
            .setRetryTimes(3)
            .setSleepTime(0)
            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36 Edg/83.0.478.56");

    @Override
    public Site getSite() {
        return site;
    }

    /**
     * 设置拼接的url变量
     */
    private static int count = 2;

    private static final String urlPre = "https://www.acfun.cn";

    public static void main(String[] args) {
        Spider.create(new AcFunJobProcessor())
                .addUrl("https://www.acfun.cn/search?type=video&keyword=%E7%BE%8E%E5%A5%B3")
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
                .thread(1)
                .run();
    }
}
